# -*- coding: utf-8 -*-
import requests
from lxml import etree
import re
import time
from loguru import logger
import random
import datetime
import execjs
import threading
import traceback
import sys
import os
import json
from queue import Queue, Empty
from fake_useragent import UserAgent
import concurrent.futures
from curl_cffi import requests as curl_requests
from src.service.requests_middler import requests_sys
from src.service.requests_middler import proxy_requests_sys
from src.service.requests_middler import insert_shipping_info
from src.service.requests_middler import read_excel
from src.service.requests_middler import read_transport_json
from src.service.requests_middler import get_proxies
ip_queue = Queue()
current_working_directory = os.getcwd()
ua = UserAgent()
random_user_agent = ua.random
stop_threading_event = threading.Event()

total_requests = 0
loss_requests = 0

def proxies_queue():
    # 获取代理并放入队列
    if ip_queue.empty():
        proxy = get_proxies()
        logger.info(f"获取ip代理成功: {proxy}")
        if proxy:
            ip_queue.put(proxy)
        else:
            logger.error(f"获取ip代理失败")
    else:
        proxy = ip_queue.get()
        return proxy

def get_Appkey():
    global total_requests
    global loss_requests
    headers = {
        "Accept": "application/json, text/plain, */*",
        "Accept-Language": "zh-CN,zh;q=0.9",
        "Cache-Control": "no-cache",
        "Connection": "keep-alive",
        "Pragma": "no-cache",
        "Referer": "https://moc.oocl.com/nj_prs_wss/",
        "Sec-Fetch-Dest": "empty",
        "Sec-Fetch-Mode": "cors",
        "Sec-Fetch-Site": "same-origin",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
        "sec-ch-ua": "\"Google Chrome\";v=\"129\", \"Not=A?Brand\";v=\"8\", \"Chromium\";v=\"129\"",
        "sec-ch-ua-mobile": "?0",
        "sec-ch-ua-platform": "\"Windows\""
    }
    url = "https://moc.oocl.com/nj_prs_wss/CSCaptchaConfig"
    retry_count = 0
    while retry_count < 3:
        res = requests_sys(url, headers=headers, GET=True)
        if res:
            js_data = res.json()
            if js_data.get("normalAppKey"):
                Appkey = js_data['normalAppKey']
                return Appkey
            else:
                logger.error(f"获取APPkey秘钥失败")
                retry_count += 1
                continue
        elif res == 0:
            retry_count += 1
            loss_requests += 1

def get_repData():
    global total_requests
    global loss_requests
    headers = {
        "Accept": "application/json, text/plain, */*",
        "Accept-Language": "zh-CN,zh;q=0.9",
        "Cache-Control": "no-cache",
        "Connection": "keep-alive",
        "Origin": "https://moc.oocl.com",
        "Pragma": "no-cache",
        "Referer": "https://moc.oocl.com/",
        "Sec-Fetch-Dest": "empty",
        "Sec-Fetch-Mode": "cors",
        "Sec-Fetch-Site": "cross-site",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
        "sec-ch-ua": "\"Google Chrome\";v=\"129\", \"Not=A?Brand\";v=\"8\", \"Chromium\";v=\"129\"",
        "sec-ch-ua-mobile": "?0",
        "sec-ch-ua-platform": "\"Windows\""
    }
    url = "https://cs-captcha-public.cargosmart.com/captcha/public/get"
    # appKey = get_Appkey()
    params = {
        "appKey": "1dc597b617744cb49c97e20b523931e1",
        "captchaType": "blockPuzzle"
    }
    retry_count = 0
    while retry_count < 3:
        res = requests_sys(url, headers=headers, params=params, GET=True)
        if res:
            js_data = res.json()
            if js_data.get("success"):
                return js_data['repData']
            else:
                logger.error(f"获取repData失败")
                retry_count += 1
                continue
        elif res == 0:
            loss_requests += 1
            retry_count += 1

def get_point(Pars):
    global total_requests
    global loss_requests
    headers = {
        "accept": "*/*",
        "accept-language": "zh-CN,zh;q=0.9",
        "cache-control": "no-cache",
        "pragma": "no-cache",
        "priority": "u=1, i",
        "referer": "https://www.oocl.com/eng/Pages/default.aspx",
        "sec-ch-ua": "\"Google Chrome\";v=\"129\", \"Not=A?Brand\";v=\"8\", \"Chromium\";v=\"129\"",
        "sec-ch-ua-mobile": "?0",
        "sec-ch-ua-platform": "\"Windows\"",
        "sec-fetch-dest": "empty",
        "sec-fetch-mode": "cors",
        "sec-fetch-site": "same-origin",
        "user-agent": random_user_agent,
        "x-requested-with": "XMLHttpRequest"
    }
    url = "https://www.oocl.com/_catalogs/masterpage/AutoCompleteSailingSchedule.aspx"
    params = {
        "type": "sailingSchedule",
        "Pars": Pars
    }
    # print(impersonate)
    # proxies = proxies_queue()
    res = curl_requests.get(url, headers=headers, params=params, impersonate="chrome124")
    # res = curl_requests.get(url, headers=headers, params=params, impersonate="chrome124", proxies=proxies)
    if res.status_code == 200:
        js_data = res.json()
        # ip_queue.put(proxies)
        if js_data.get("data").get("results"):
            return js_data['data']['results']
        else:
            logger.error(f"{Pars}: 查询失败")
            return None
    else:
        logger.error(f"{url} 请求失败: {res.status_code}")
        return None

def minutes_to_days(minutes):
    hours = minutes / 60          # 将分钟转换为小时
    days = hours / 24             # 将小时转换为天
    return int(days)


def load_mapping(file_name):
    """加载现有的城市映射关系"""
    if os.path.exists(file_name):
        with open(file_name, 'r', encoding='utf-8') as file:
            return json.load(file)
    return {}

def save_mapping(file_name, mapping):
    """保存城市映射关系到文件"""
    with open(file_name, 'w', encoding='utf-8') as file:
        json.dump(mapping, file, ensure_ascii=False, indent=4)

def oocl_transport(origin_city, destination_city):
    try:
        global total_requests
        global loss_requests
        port_mapping = {}
        # file_path = "maeu_ports.json"
        # file_path = os.path.join(current_working_directory + r'/src/crawler', file_path)
        file_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'maeu_ports.json')
        if os.path.exists(file_path):
            with open(file_path, "r") as file:
                Menu_json = json.load(file)
        else:
            logger.error(f"不存在文件: {file_path}")
            sys.exit()
        for info in Menu_json['ports']:
            port_mapping[info['portName'].upper()] = {
                'portCode': info.get('portCode'),
                'unLocationCode': info['unLocationCode'],
                'regionCode': info.get('regionCode')
            }
        origin_info = port_mapping.get(origin_city.upper())
        destination_info = port_mapping.get(destination_city.upper())
        if origin_info and destination_info:
            total_requests += 1
            date = datetime.datetime.now().strftime('%Y-%m')
            origin_unLocationCode = origin_info['unLocationCode']
            des_unLocationCode = destination_info['unLocationCode']
            headers = {
                "Accept": "application/json, text/plain, */*",
                "Accept-Language": "zh-CN,zh;q=0.9",
                "Cache-Control": "no-cache",
                "Connection": "keep-alive",
                "Content-Type": "application/json",
                "Origin": "https://moc.oocl.com",
                "Pragma": "no-cache",
                "Referer": "https://moc.oocl.com/nj_prs_wss/",
                "Sec-Fetch-Dest": "empty",
                "Sec-Fetch-Mode": "cors",
                "Sec-Fetch-Site": "same-origin",
                "User-Agent": random_user_agent,
                "sec-ch-ua": "\"Google Chrome\";v=\"129\", \"Not=A?Brand\";v=\"8\", \"Chromium\";v=\"129\"",
                "sec-ch-ua-mobile": "?0",
                "sec-ch-ua-platform": "\"Windows\""
            }
            url = "https://moc.oocl.com/nj_prs_wss/mocss/secured/supportData/nsso/searchHubToHubRoute"
            js_file_path = 'OOCL.js'

            js_file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), js_file_path)
            with open(js_file_path, 'r', encoding='utf-8')as f:
                js_code = f.read()
            ctx = execjs.compile(js_code)
            repData = get_repData()
            # print(repData)
            captchaToken = ctx.call("get_token", repData)
            # print(csToken)
            # 加载现有的城市映射关系
            # json_file_name = 'OOCL_query.json'
            json_file_name = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'OOCL_query.json')

            city_mapping = load_mapping(json_file_name)
            # print(f"城市映射关系{city_mapping}")
            # 获取源城市的映射
            if origin_city not in city_mapping:
                origin_query = get_point(origin_city)
                if origin_query:
                    origin_query_sys = origin_query[0]
                    city_mapping[origin_city] = origin_query_sys
            else:
                origin_query_sys = city_mapping[origin_city]

            # 获取目标城市的映射
            if destination_city not in city_mapping:
                des_query = get_point(destination_city)
                if des_query:
                    des_query_sys = des_query[0]  # 假设我们只关心第一个结果
                    city_mapping[destination_city] = des_query_sys
            else:
                des_query_sys = city_mapping[destination_city]

            # 保存更新后的映射关系
            save_mapping(json_file_name, city_mapping)
            # origin_query_sys = None
            # origin_query = get_point(origin_city)
            # for info in origin_query:
            #     for info2 in info['Codes']:
            #         if origin_unLocationCode.upper() == info2['Code'].upper():
            #             origin_query_sys = info
            #             break
            #         else:
            #             origin_query_sys = info
            # des_query_sys = None
            # des_query = get_point(destination_city)
            # for info in des_query:
            #     for info2 in info['Codes']:
            #         if des_unLocationCode.upper() == info2['Code'].upper():
            #             des_query_sys = info
            #             break
            # print(origin_query_sys)
            # print(des_query_sys)
            # origin_query_sys, des_query_sys = query_location(origin_city, origin_unLocationCode, destination_city, des_unLocationCode)
            if origin_query_sys and des_query_sys:
                date = datetime.datetime.now().strftime('%Y-%m-%d')
                data = {
                    "date": f"{date}T00:00:00.000Z", # f"{date}-01T00:00:00.000Z"
                    "displayDate": f"{date}T00:00:00.000Z", # f"{date}-01T00:00:00.000Z"
                    "transhipment_Port": None,
                    "port_of_Load": None,
                    "port_of_Discharge": None,
                    "sailing": "sailing.from",
                    "weeks": "4",
                    "transhipment_PortId": None,
                    "service": None,
                    "port_of_LoadId": None,
                    "port_of_DischargeId": None,
                    "origin_Haulage": "cy",
                    "destination_Haulage": "cy",
                    "cargo_Nature": "dry",
                    "originId": origin_query_sys["LocationID"],
                    "originCountryCode": "CN",
                    "destinationCountryCode": "US",
                    "originCityTimeZone": origin_query_sys["OtherDetails"][0]["Value"],
                    "destinationCityTimeZone": des_query_sys["OtherDetails"][0]["Value"],
                    "destinationId": des_query_sys['LocationID'],
                    "origin": origin_query_sys["Names"][0]["Name"] + ',' + origin_query_sys["Names"][0]["UpperAdministrativeLocation"],
                    "destination": des_query_sys["Names"][0]["Name"] + ',' + des_query_sys["Names"][0]["UpperAdministrativeLocation"],
                    # "showCaptcha": "false",
                    "captchaToken": captchaToken,
                    "captchaAppKey": "1dc597b617744cb49c97e20b523931e1",
                    "weeksSymbol": "+"
                }
                data = json.dumps(data, separators=(',', ':'))
                retry_count = 0
                res = requests_sys(url, headers=headers, data=data, POST=True)
                if res:
                    js_data = res.json()
                    if js_data.get("errorInfo") == "Captcha Error! Please try again.":
                        logger.error(f"验证码请求失败")
                    logger.info(f"当前查询 起点:{origin_city} ----> 终点: {destination_city}")
                    logger.info(f"当前请求总数: {total_requests}, 失败请求总数: {loss_requests}")
                    total_nums = js_data["data"]["numberOfRouteReturn"]
                    logger.info(f"当前查询 起点:{origin_city} ----> 终点: {destination_city} 总数为： {total_nums}")
                    total_list = []
                    if js_data.get("data").get("standardRoutes"):
                        for info in js_data["data"]["standardRoutes"]:
                            for i in range(min(4, len(info["Legs"]))):
                                if "VesselName" in info["Legs"][i]:
                                    num = i
                                    break  # 找到后退出循环
                            item = {}
                            item['origin_code'] = f"{origin_city}({origin_unLocationCode})"

                            item['transport_contractor'] = "OOCL"

                            # FromETDLocalDateTime下的最后一个是结束时间 / 第一个CargoAvailabilityLocalDateTime是出发时间
                            # item['depart_date'] = datetime.datetime.strptime('20241019070000.000', "%Y%m%d%H%M%S.%f").strftime("%Y-%m-%d")
                            item['depart_date'] = datetime.datetime.strptime(info["Legs"][int(num)]["FromETDLocalDateTime"]['dateStr'], "%Y%m%d%H%M%S.%f").strftime("%Y-%m-%d")
                            if len(info['Legs']) == 3:
                                # Savannah
                                # print(info)
                                discharge_port_name = info['Legs'][int(num)]['DischargePort']['Name'].upper()
                                port_info = port_mapping.get(discharge_port_name)
                                if port_info is not None:
                                    # 增加州代码/
                                    # 获取中转站编码
                                    # transfer_code = info['Legs'][-1]["City"]['Name'].split(',')[1].strip()
                                    transfer_code = info['Legs'][int(num)]['DischargePort']['Name']  # 获取中转站
                                    if discharge_port_name != transfer_code.upper():
                                        item['transfer_code'] = transfer_code + f"({port_mapping.get(transfer_code.upper())['unLocationCode']})"
                                    else:
                                        item['transfer_code'] = None
                                    regionCode = port_mapping.get(discharge_port_name)['regionCode']
                                    if regionCode is None:
                                        destination_code = f"{discharge_port_name},{regionCode}({port_info.get('unLocationCode')})"
                                    else:
                                        destination_code = f"{discharge_port_name}({port_info.get('unLocationCode')})"
                                else:
                                    destination_code = f"{discharge_port_name}(Unknown Code)"

                                item['destination_code'] = destination_code
                                item['arrive_date'] = datetime.datetime.strptime(info["Legs"][int(num)]["ToETALocalDateTime"]['dateStr'], "%Y%m%d%H%M%S.%f").strftime("%Y-%m-%d")
                                item['haulage_time'] = minutes_to_days(int(info['Legs'][int(num)]['TransitTimeInMinute']))
                                item['ship_name'] = info["Legs"][int(num)]["VesselName"]
                                item['voyage_number'] = info["Legs"][int(num)]["ExternalVoyageReference"]
                                total_list.append(item)
                            elif len(info['Legs']) == 4:
                                # Atlanta
                                # print(info)
                                transfer_code = info['Legs'][int(num)]['DischargePort']['Name'] # 获取中转站
                                # 目的地
                                destination_code = info['Legs'][-1]['City']['Name'].split(',')[0].upper()
                                if transfer_code.upper() != destination_code:
                                    # tr_code = port_mapping.get(transfer_code.upper())['unLocationCode']
                                    # 中转站编码

                                    item['transfer_code'] = transfer_code + f"({port_mapping.get(transfer_code.upper())['unLocationCode']})"
                                else:
                                    item['transfer_code'] = None

                                # 将 destination_code 转换为大写以确保一致性
                                destination_code_upper = destination_code.upper()
                                city_mapping_lower = {key.lower(): value for key, value in city_mapping.items()}
                                destination_value = city_mapping_lower.get(destination_code_upper.lower())

                                # 假设数据结构中有 'Codes' 键
                                codes = destination_value.get('Codes', [])
                                for code in codes:
                                    if len(code['Code']) == 5 and code['Code'].isalpha():
                                        des_unLocationCode2 = code['Code']
                                # 增加目的地，州代码
                                regionCode = port_mapping.get(f"{destination_value['Names'][0]['Name'].upper()}")['regionCode']
                                if regionCode is None:
                                    # 州代码为空就
                                    item['destination_code'] = f"{destination_value['Names'][0]['Name']}({des_unLocationCode2})"
                                else:
                                    item['destination_code'] = f"{destination_value['Names'][0]['Name']},{regionCode}({des_unLocationCode2})"
                                item['arrive_date'] = datetime.datetime.strptime(info["Legs"][int(num + 1)]["CargoAvailabilityLocalDateTime"]['dateStr'],"%Y%m%d%H%M%S.%f").strftime("%Y-%m-%d")
                                total_transit_time = sum(int(leg['TransitTimeInMinute']) for leg in info['Legs'])
                                item['haulage_time'] = minutes_to_days(total_transit_time)
                                item['ship_name'] = info["Legs"][int(num)]["VesselName"]
                                item['voyage_number'] = info["Legs"][int(num)]["ExternalVoyageReference"]
                            else:
                                # print("网页结构:", len(info["Legs"]))
                                # 目的地
                                destination_code = info['Legs'][-1]['City']['Name'].split(',')[0].upper()
                                ##########
                                # 多个中转站
                                tmp_legs = []
                                for legs in info['Legs']:
                                    if "VesselName" in legs:
                                        tmp_legs.append(legs)
                                if len(tmp_legs) > 1:
                                    transfer_code = tmp_legs[-1]['DischargePort']['Name']
                                else:
                                    transfer_code = tmp_legs[0]['DischargePort']['Name']
                                if transfer_code.upper() != destination_code:
                                    item['transfer_code'] = transfer_code + f"({port_mapping.get(transfer_code.upper())['unLocationCode']})"
                                else:
                                    item['transfer_code'] = None
                                destination_code_upper = destination_code.upper()
                                city_mapping_lower = {key.lower(): value for key, value in city_mapping.items()}
                                destination_value = city_mapping_lower.get(destination_code_upper.lower())
                                codes = destination_value.get('Codes', [])
                                for code in codes:
                                    if len(code['Code']) == 5 and code['Code'].isalpha():
                                        des_unLocationCode2 = code['Code']
                                regionCode = port_mapping.get(f"{destination_value['Names'][0]['Name'].upper()}")['regionCode']
                                if regionCode is None:
                                    item['destination_code'] = f"{destination_value['Names'][0]['Name']}({des_unLocationCode2})"
                                else:
                                    item['destination_code'] = f"{destination_value['Names'][0]['Name']},{regionCode}({des_unLocationCode2})"
                                item['arrive_date'] = datetime.datetime.strptime(
                                info["Legs"][int(-1)]["CargoCutoffLocalDateTime"]['dateStr'],
                                        "%Y%m%d%H%M%S.%f").strftime("%Y-%m-%d")
                                item['depart_date'] = datetime.datetime.strptime(info['Legs'][0]['CargoAvailabilityLocalDateTime']['dateStr'], "%Y%m%d%H%M%S.%f").strftime("%Y-%m-%d")
                                total_transit_time = int(info['TransitTimeInMinute']) # 取到总分钟
                                item['haulage_time'] = minutes_to_days(total_transit_time)
                                item['ship_name'] = info["Legs"][int(num)]["VesselName"]
                                item['voyage_number'] = info["Legs"][int(num)]["ExternalVoyageReference"]
                            item['shipping_lanes'] = info["Legs"][int(num)]["Service"]
                            # insert_shipping_info(item)
                            print(item)
                    else:
                        logger.info(f"起点:{origin_city} ----> 终点: {destination_city} 获取数据失败")
                        read_transport_json(transport='OOCL', origin_city=origin_city, des_city=destination_city,
                                       fail_result="获取数据失败")
                        loss_requests += 1
                elif res == 0:
                    logger.info(f"起点:{origin_city} ----> 终点: {destination_city} 获取数据失败")
                    read_transport_json(transport='OOCL', origin_city=origin_city, des_city=destination_city,
                                   fail_result="请求超时")
                    loss_requests += 1
            else:
                logger.info(f"起点:{origin_city} ----> 终点: {destination_city} 查询数据失败")
                read_transport_json(transport='OOCL', origin_city=origin_city, des_city=destination_city,
                               fail_result="查询数据失败")
                loss_requests += 1
    except Exception as e:
        logger.error(f"{origin_city} ----> {destination_city}: {e}")
        print(traceback.format_exc())
#OOCL

def process_route(origin_city, destination_list):
    for destination_city in destination_list:
        oocl_transport(origin_city, destination_city)

def oocl_spider_run():
    global total_requests
    global loss_requests
    # origin_city_list = [
    #     "Shanghai", "XIAMEN", "NANSHA", "QINGDAO", "YANTIAN", "NINGBO"
    # ]
    origin_city_list = [
        "Shanghai"
    ]
    #
    destination_list = [
        "Los Angeles", "Long Beach", "Atlanta", "Baltimore", "Boston", "Buffalo", "Calgary", "Charleston",
        "Chicago", "Chippewa Falls", "Cincinnati", "Cleveland", "Dallas", "Denver",
        "Detroit", "Edmonton", "EL Paso", "Halifax", "Houston", "Jacksonville",
        "Kansas City", "Louisville", "Memphis", "Miami", "Minneapolis", "Montreal",
        "Nashville", "New Orleans", "New York", "Norfolk", "Oakland", "Omaha",
        "Philadelphia", "Pittsburgh", "Portland", "Prince Rupert", "Saint Louis",
        "Salt Lake City", "San Antonio", "Saskatoon", "Savannah", "Seattle",
        "Tacoma", "Tampa", "Toronto", "Vancouver", "Winnipeg"
    ]

    # with concurrent.futures.ThreadPoolExecutor() as executor:
    #     futures = [executor.submit(process_route, origin_city, destination_list) for origin_city in origin_city_list]
    # json_path_one = r"origin_and_destination.json"
    # # current_working_directory_one = os.getcwd() + "/src/crawler"
    # # json_path = os.path.join(current_working_directory_one, json_path_one)
    # json_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), json_path_one)
    # with open(f'{json_path}', 'r', encoding='utf-8') as f:
    #     tmp = json.loads(f.read())
    #
    # origin_city_list = tmp['origin_city_list']
    # destination_list = tmp['destination_list']
    # del tmp, json_path
    with concurrent.futures.ThreadPoolExecutor() as executor:
        futures = []
        for origin_city in origin_city_list:
            future = executor.submit(process_route, origin_city, destination_list)
            futures.append(future)
            time.sleep(5)

        for future in concurrent.futures.as_completed(futures):
            try:
                future.result()
            except Exception as e:
                # 输出异常详细信息，包括出错的行号
                print(f"An error occurred: {e}")
                print(traceback.format_exc())
    read_excel(transport="OOCL", total_requests=total_requests, loss_requests=loss_requests)

if __name__ == '__main__':
    oocl_spider_run()